#!/usr/bin/python2.7


################################################################################
# Copyright (c) 2012 Evangelos Kanoulas
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
# THE SOFTWARE.
################################################################################

"""Tests session evaluation script.
"""

__author__ = 'ekanoulas@gmail.com (Evangelos Kanoulas)'

import unittest

import session_eval as session

class QrelReaderTest(unittest.TestCase):
  def setUp(self):
    self.qrel_file = 'testing/qrels'
    self.mapping = {'-2':'0', '0':'0', '1':'1', '4':'2', '2':'3', '3':'4'}

  def test_read_qrel(self):
    qrelreader = session.QrelReader(self.qrel_file)
    qrels = qrelreader.ReadQrels()
    self.assertEqual(qrels['46']['clueweb09-en0000-05-13273'], 4)
    self.assertEqual(qrels['46']['clueweb09-en0000-95-27473'], -2)

  def test_map_judgments(self):
    qrelreader = session.QrelReader(self.qrel_file)
    qrels = qrelreader.ReadQrels()
    qrels = qrelreader.MapJudgmentsToGrades(qrels, self.mapping)
    self.assertEqual(qrels['46']['clueweb09-en0000-05-13273'], 2)
    self.assertEqual(qrels['46']['clueweb09-en0000-95-27473'], 0)


class RankingReaderTest(unittest.TestCase):
  def setUp(self):
    self.qrel_file = 'testing/qrels'
    self.mapping = {'-2':'0', '0':'0', '1':'1', '4':'2', '2':'3', '3':'4'}
    self.mapping_file = 'testing/mapping.txt'
    self.run = 'testing/run.txt'
    self.relevance = [1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1]

  def test_read_ranking(self):
    rankingreader = session.RankingReader(self.mapping_file, self.run)
    ranking_docs = rankingreader.ReadRanking()
    self.assertEqual(ranking_docs['79'][0], 'clueweb09-en0004-39-12399')
    
  def test_map_doc_to_rel(self):
    qrelreader = session.QrelReader(self.qrel_file)
    qrels = qrelreader.ReadQrels()
    qrels = qrelreader.MapJudgmentsToGrades(qrels, self.mapping)
    self.assertEqual(qrels['46']['clueweb09-en0000-05-13273'], 2)
    rankingreader = session.RankingReader(self.mapping_file, self.run)
    ranking_docs = rankingreader.ReadRanking()
    ranking = rankingreader.MapDocumentsToRelevance(ranking_docs, qrels)
    self.assertEqual(ranking['79'], self.relevance)

class MeasureEvaluatorTest(unittest.TestCase):
  def setUp(self):
    self.ranking = [0, 0, 0, 1, 2, 3, 0, 0, 1, 0, 2, 0, 1]
    self.cutoff_a = 1
    self.cutoff_b = 10
    self.cutoff_c = 20
    self.R = 10

  def test_measures(self):
    measureeval = session.MeasureEvaluator()
    self.assertEqual(measureeval.PrecisionAtK(self.ranking, self.cutoff_a), 0)
    self.assertEqual(measureeval.PrecisionAtK(self.ranking, self.cutoff_b), 0.4)
    self.assertEqual(measureeval.PrecisionAtK(self.ranking, self.cutoff_c), 0.3)
    self.assertAlmostEqual(measureeval.AveragePrecision(self.ranking, self.R),
                           0.25105, delta=0.0001)
    self.assertAlmostEqual(measureeval.ERR(self.ranking),
                           0.1177147, delta=0.0001)
    self.assertAlmostEqual(measureeval.DCG(self.ranking),
                           5.48519, delta=0.0001)
    self.assertAlmostEqual(measureeval.DCG(self.ranking, self.cutoff_a),
                           0, delta=0.0001)
    self.assertAlmostEqual(measureeval.DCG(self.ranking, self.cutoff_b),
                           4.38571, delta=0.0001)
    self.assertAlmostEqual(measureeval.DCG(self.ranking, self.cutoff_c),
                           5.48519, delta=0.0001)

class SessionEvaluatorTest(unittest.TestCase):
  def setUp(self):
    self.mapping_file = 'testing/mapping.txt'
    self.qrel_file = 'testing/qrels'
    self.run = 'testing/run.txt'
    self.relevance = [1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1]
    self.mapping = {'-2':'0', '0':'0', '1':'1', '4':'2', '2':'3', '3':'4'}
    self.grades = list(set(map(int, self.mapping.values())))

  def test_generate_ideal_ranking(self):
    qrelreader = session.QrelReader(self.qrel_file)
    qrels = qrelreader.ReadQrels()
    qrels = qrelreader.MapJudgmentsToGrades(qrels, self.mapping)
    self.assertEqual(qrels['46']['clueweb09-en0000-05-13273'], 2)

    rankingreader = session.RankingReader(self.mapping_file, self.run)
    ranking_docs = rankingreader.ReadRanking()
    ranking = rankingreader.MapDocumentsToRelevance(ranking_docs, qrels)
    self.assertEqual(ranking['79'], self.relevance)

    sessionevaluator = session.SessionEvaluator(self.mapping_file, qrels,
                                                ranking, self.grades)
    self.assertEqual(sessionevaluator.ideal_ranking['91'],
                     [3, 2, 2, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0])
    self.assertEqual(sessionevaluator.relevance_counts['91']['rel'], 6)
    measures = sessionevaluator.ComputeMeasures()
    self.assertEqual(measures['precision_at_k']['79'], 0.6)

if __name__ == '__main__':
  unittest.main()
